package com.example.myFirstWebmagic.dailyAirReport.processor;

import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;

import com.example.myFirstWebmagic.dailyAirReport.entity.AirQualityReportPK;
import com.example.myFirstWebmagic.dailyAirReport.entity.EnDailyAirQualityReport;
import com.example.myFirstWebmagic.pipeline.ConsolePipelineDemo;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

public class NewAirDailyProcessor implements PageProcessor{

	
	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private Site site = Site.me().setRetryTimes(10).setSleepTime(5000);
    //所有的地市
    private static List<String> cities=new ArrayList<String>();
    //页面上的开始时间
    private String startime="";
    //访问连接
    private String targetUrlTemp="http://datacenter.mep.gov.cn:8099/ths-report/report!list.action?xmlname=1464591308421&";
    
    //目标连接
    private List<String> targetUrls=new ArrayList<String>();
    
    private List<EnDailyAirQualityReport> qualityReports=new ArrayList<EnDailyAirQualityReport>();
    
	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String [] startUrl=new String[]{"http://datacenter.mep.gov.cn/index!getCityMain.action"};
		// TODO Auto-generated method stub
		Spider.create(new NewAirDailyProcessor())
        //从"http://datacenter.mep.gov.cn/index!getCityMain.action"开始抓
        .addUrl(startUrl)
        .addPipeline(new ConsolePipelineDemo())
        //开启5个线程抓取
        .thread(5)
        //启动爬虫
        .run();
		
	}

	public void process(Page page){
		// TODO Auto-generated method stub
		//获取所有的地市名称
		
		cities=page.getHtml().xpath("//div[@class='aqi_list']//p//span[@class='clickcss']/text()").all();
		//获取开始时间
		this.setStartime(page.getHtml().xpath("//input[@id='start']/@value").toString());
		
		try {
			for(String city:cities){
				targetUrls.add(targetUrlTemp+"starttime="+getStartime()+"&cityname="+city);
			}
		} catch (Exception e) {
			// TODO: handle exception
		}
			
		//日期
		List<String> dates= page.getHtml().xpath("//*[@id='GridView1']/tbody//tr/td[2]/span/text()").all();
		//城市
		List<String> citys= page.getHtml().xpath("//*[@id='GridView1']/tbody//tr/td[3]/span/text()").all();
//		//aqi指数
		List<String> aqis= page.getHtml().xpath("//*[@id='GridView1']/tbody//tr/td[4]/span/text()").all();
//		//空气质量级别
		List<String> airLevels= page.getHtml().xpath("//*[@id='GridView1']/tbody//tr/td[7]/span/text()").all();
//		//污染物
		List<String> populates= page.getHtml().xpath("//*[@id='GridView1']/tbody//tr/td[5]/span/text()").all();
		

		
		
		for(String date:dates){
			int index=dates.indexOf(date);
			AirQualityReportPK airQualityReportPK=new AirQualityReportPK(citys.get(index), dates.get(index));
			EnDailyAirQualityReport airQualityReport=new EnDailyAirQualityReport(airQualityReportPK, aqis.get(index), airLevels.get(index), populates.get(index));
			qualityReports.add(airQualityReport);
		}
		
		
		page.putField("objectList", qualityReports);
		
		page.addTargetRequests(targetUrls);	
	}


	public Site getSite() {
		// TODO Auto-generated method stub
		return site;
	}

	public String getStartime() {
		return startime;
	}

	public void setStartime(String startime) {
		if(StringUtils.isEmpty(this.startime)){
			this.startime=startime;
		}
	}

	
}
